package hutool.url;

import cn.hutool.core.util.ReUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpRequest;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Xiongx
 * @version 1.0
 * @date 2021/5/22 23:17
 * @since JDK 1.8
 */
public class Url {

    public static void main(String[] args) {
        new Url().Demo6();
    }



    //栏目主页爬取
    public void Demo6(){
        String httpBase="";
        String body=HttpRequest.get(httpBase+"/index.html").execute().body();
        String channelMain=ReUtil.get("<ul class=\"mainnav\">(.*?)</ul>",body,1);  //获取栏目核心主体部分
        List<String> mainChannelList=ReUtil.findAll("<b>(.*?)</b>",channelMain,1);
        for (String string:mainChannelList){
            Map<String, String> stringStringMap=getStringMap(string,"a","href");
            System.out.println("=======================================================================");
            String href=stringStringMap.get("href");
            String channelName=stringStringMap.get("tag");
            System.out.println("href: "+href);
            System.out.println("channelName："+channelName);
            if("首页".equals(channelName)){
                continue;
            }
            String body2=HttpRequest.get(httpBase+href).execute().body();
            System.out.println("----------------------所有三级栏目-------------------------");
            String channelBody=ReUtil.get("<div class=\"Menu\">(.*?)</div>",body2,1);
            //System.out.println(channelBody);
            List<String> channelList=ReUtil.findAll("<li .*?>(.*?)</li>",channelBody,1);
            if(channelList==null){
                continue;
            }
            for (String string2:channelList){
                Map<String, String> stringStringMap1=getStringMap(string2,"a","href");
                String href1=stringStringMap1.get("href");
                String channelName2=stringStringMap1.get("tag");
                System.out.println("href: "+href1);
                System.out.println("channelName："+channelName2);
                if("".equals(href1)){
                    continue;
                }
                System.out.println("################  开始获取列表  #######################");
                if(href1.contains("www")||href1.contains("java")){
                    continue;
                }
                href1=href1.replace(".html","");
                String listbody=HttpRequest.get(httpBase+href1+"_page_"+1+".html").execute().body();
                String body3=ReUtil.get("<ul class=\"ul_li list_news\">(.*?)</ul>",listbody,1);  //获取核心主体部分
                List<String> titleList=ReUtil.findAll("<li>(.*?)</li>",body3,1);
                if(titleList==null){
                    continue;
                }
                System.out.println("------------------------------------------------------");
                for (String title:titleList){
                    String date=ReUtil.get("<span>(.*?)</span>",title,1);
                    Map<String, String> stringMap=getStringMap(title,"a","href");
                    String href4=stringMap.get("href");
                    String tag=stringMap.get("tag");
                    System.out.println("date: "+date);
                    System.out.println("title: "+tag);
                    if(href4.contains("www")||href4.contains("java")){
                        continue;
                    }
                    String json=HttpRequest.get(httpBase+href4).execute().body();
                    //使用正则获取所有标题
                    String content=ReUtil.get("<div class=\"info_txt\">(.*?)</div>",json, 1);
                    String sourceName=ReUtil.get("<span class=\"source\">(.*?)</span>",json, 1);
                    System.out.println("content: "+content);
                    System.out.println("sourceName: "+sourceName);
                    System.out.println("-------------------------------------------------------------------------------");
                }
                System.out.println("##########################################################################");
            }
        }

    }




    //列表页爬取
    public void Demo5(String path,Integer code,Integer page){
        String httpBase="";
        String listbody=HttpRequest.get(httpBase+path+"/list_"+code+"_page_"+page+".html").execute().body();
        String body=ReUtil.get("<ul class=\"ul_li list_news\">(.*?)</ul>",listbody,1);  //获取核心主体部分
        List<String> titleList=ReUtil.findAll("<li>(.*?)</li>",body,1);
        for (String title:titleList){
            String date=ReUtil.get("<span>(.*?)</span>",title,1);
            Map<String, String> stringMap=getStringMap(title,"a","href");
            String href=stringMap.get("href");
            String tag=stringMap.get("tag");
            System.out.println("===================================================");
            System.out.println("date: "+date);
            System.out.println("title: "+tag);
            System.out.println("href: "+href);
            System.out.println("---------------------------------------------------");
            String json=HttpRequest.get(httpBase+href).execute().body();
            //使用正则获取所有标题
            String content=ReUtil.get("<div class=\"info_txt\">(.*?)</div>",json, 1);
            String sourceName=ReUtil.get("<span class=\"source\">(.*?)</span>",json, 1);
            System.out.println("content: "+content);
            System.out.println("sourceName: "+sourceName);
            System.out.println("====================================================");
        }
    }

    //获取列表
    public void Demo4(){
        String list=HttpRequest.get("").execute().body();
        String body=ReUtil.get("<ul class=\"ul_li list_news\">(.*?)</ul>",list,1);
        List<String> stringList=ReUtil.findAll("<li>(.*?)</li>",body,1);
        stringList.forEach(e->{
            System.out.println("========================================");
            String date=ReUtil.get("<span>(.*?)</span>",e,1);
           // getStringList(e);
            //String href=ReUtil.get("<a [^<>]*?\\s href =['\"]?(.*?)['\"]?(\\s.*?)?>([^><]*)</a>",e,1);
            System.out.println(date); //日期
          //  System.out.println(href);
            System.out.println(e);
        });
    }

    public Map<String, String> getStringMap(String html,String tag,String tagAttr) {
        Map<String, String> stringStringMap=new HashMap<>(2);
        String reg = "<" + tag + "[^<>]*?\\s" + tagAttr + "=['\"]?(.*?)['\"]?(\\s.*?)?>([^><]*)</" + tag + ">";
        Matcher m = Pattern.compile(reg).matcher(html);
        while (m.find()) {
            String href = m.group(1); //tagAttr的值
            String tagContent = m.group(3);  //标签包裹的值
            stringStringMap.put("href",href);
            stringStringMap.put("tag",tagContent);
        }
        return stringStringMap;
    }

    public void Demo3(){
        String json=HttpRequest.get("").execute().body();
        //使用正则获取所有标题

        String title=ReUtil.get("<h2 class=\"info_title\">(.*?)</h2>",json,1);
        String content=ReUtil.get("<div class=\"info_txt\">(.*?)</div>",json, 1);
        String sourceName=ReUtil.get("<span class=\"source\">(.*?)</span>",json, 1);

        System.out.println("title: "+title);
        System.out.println("content: "+content);
        System.out.println("sourceName: "+sourceName);


    }

    //解码
    public void Demo2(String string){
        String decode=URLUtil.decode(string);
        System.out.println(decode);
    }
}